#coding: utf-8
#Encoding.default_external = Encoding::UTF_8
#gem 'rails','~>3.2.7'
require 'rubygems'
require 'hpricot'
require 'open-uri'
require 'date'
require 'httpclient'
require 'nokogiri'
require 'cgi'
require 'base64'
require 'json'
require 'digest/sha1'
require 'redis'
require 'time'
#require 'iconv'

class Ipin
	OVERTIMER = 1 * 12 * 3600

	def self.get_a(cookie_path)
		a = HTTPClient.new
		a.set_cookie_store(cookie_path)
		return a
	end

	def self.set_cookie(cookie_file_path,login_email,pass)

		a = HTTPClient.new
		a.set_cookie_store(cookie_file_path)
		a.agent_name = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36"

		userlogin = login_email.gsub(/@/,"%40")
		user_b64 = Base64.strict_encode64(userlogin)
		cookie_file_path = get_cookie_path(login_email)

			prelogin_url = "http://login.sina.com.cn/sso/prelogin.php?entry=weibo&callback=sinaSSOController.preloginCallBack&su=#{user_b64}&rsakt=mod&checkpin=1&client=ssologin.js(v1.4.11)&_=#{Time.now.to_i}100"
			header = {}
			header["Accept" => "*/*"]
			header["Accept-Encoding" => "gzip,deflate,sdch"]
			header["Connection" => "keep-alive"]
			header["Host" => "login.sina.com.cn"]
			header["Referer" => "http://weibo.com/"]
			header["User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36"]
			params = {}
			params["entry"] = "weibo"
			params["callback"] = "sinaSSOController.preloginCallBack"
			params["su"] = user_b64
			params["rsakt"] = "mod"
			params["checkpin"] = "1"
			params["client"] = "ssologin.js(v1.4.11)"
			params["_"] = "#{Time.now.to_i}000"
			c = a.get(prelogin_url, params, header)
			body = c.body
			pkeys = JSON.parse(body[/\{\"retcode[^\0]*\}/])
			nonce = pkeys["nonce"]
			rsakv = pkeys["rsakv"]
			pcid = pkeys["pcid"]
			servertime = pkeys["servertime"]
			pwdkey = servertime.to_s + "\t" + nonce + "\n" + pass
			pub = OpenSSL::PKey::RSA::new
			pub.e = 65537
			pub.n = OpenSSL::BN.new(pkeys['pubkey'],16)
			encry_password = pub.public_encrypt(pwdkey).unpack('H*').first


			post_login_url = "http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)"
			header = {}
			header["Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"]
			header["Accept-Encoding" => "gzip,deflate,sdch"]
			header["Connection" => "keep-alive"]
			header["Host" => "login.sina.com.cn"]
			header["Referer" => "http://weibo.com/"]
			header["User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36"]
			params = {}
			params["from"] = "weibo"
			params["savestate"] = 7
			params["useticket"] = 1
			params["pagerefer"] = ""
			params["vsnf"] = "1"
			params["su"] = user_b64
			params["service"] = "miniblog"
			params["servertime"] = Time.now.to_i
			params["nonce"] = nonce
			params["pwencode"] = "rsa2"
			params["rsakv"] = rsakv
			params["sp"] = encry_password
			params["encoding"] = "UTF-8"
			params["prelt"] = "44"
			params["url"] = "http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack"
			params["returntype"] = "META"
			c = a.get(post_login_url, params, header)
			body = c.body

			redirect_url = body.split("location.replace('")[1].split("');});}")[0]


			header = {}
			header["Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"]
			header["Accept-Encoding" => "gzip,deflate,sdch"]
			header["Connection" => "keep-alive"]
			header["Host" => "weibo.com"]
			header["Accept-Language" => "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3"]
			header["Referer" => "http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)"]
			header["User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36"]



			c = a.get(redirect_url, nil, header)

			follow_redirect_url = "http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack&sudaref=weibo.com"

			c = a.get(follow_redirect_url,nil, header)
			 puts c.body

			header = {}
			header["Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"]
			header["Accept-Encoding" => "gzip,deflate,sdch"]
			header["Connection" => "keep-alive"]
			header["Host" => "weibo.com"]
			header["Accept-Language" => "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3"]
			header["Referer" => "http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack&sudaref=weibo.com"]
			header["User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36"]
			c = a.get("http://weibo.com/guide/welcome",nil, header)
			puts c.body
			a.cookie_manager.save_all_cookies
	end
	def self.login_cookie(login_email, pass, force_update = false)
		if login_email && pass
			cookie_file_path = get_cookie_path(login_email)
			#puts Time.now - File.mtime(cookie_file_path) < OVERTIMER
			#puts  !(File.exist?(cookie_file_path) && Time.now - File.mtime(cookie_file_path) < OVERTIMER)
			if File.exist?(cookie_file_path)
				if !(Time.now.to_i - File.mtime(cookie_file_path).to_i < OVERTIMER)
					#set_cookie(cookie_file_path,login_email,pass)
				#else
					`sudo rm -f #{cookie_file_path}`
					set_cookie(cookie_file_path,login_email,pass)
				end
			else
				set_cookie(cookie_file_path,login_email,pass)
			end
		end
		return true
	end

	def self.to_utf8(unicode_string)
		unicode_string.gsub(/\\u\w{4}/) do |s|
			str = s.sub(/\\u/, "").hex.to_s(2)
			if str.length < 8
				CGI.unescape(str.to_i(2).to_s(16).insert(0, "%"))
			else
				arr = str.reverse.scan(/\w{0,6}/).reverse.select{|a| a != ""}.map{|b| b.reverse}
				hex = lambda do |s|
					(arr.first == s ? "1" * arr.length + "0" * (8 - arr.length - s.length) + s : "10" + s).to_i(2).to_s(16).insert(0, "%")
				end
				CGI.unescape(arr.map(&hex).join)
			end
		end
	end

	def self.get_cookie_path(login_email)
		dir_url = File.join(Dir.pwd,"cookie")
		if !File.directory?("#{dir_url}")
			Dir.mkdir(dir_url,0777)
		end
		cookie_file_path = "#{dir_url}/sina_cookie_#{login_email.sub(/@/,"")}"
	end

	def self.get_connection(username,pwd)
		self.login_cookie(username, pwd)
		cookie_path = self.get_cookie_path(username)
		a = get_a(cookie_path)

		heads1 = {
			"(Request-Line)"=>"GET /oauth2/authorize?scope=all&redirect_uri=http%3A//www.ipin.com/account/callback/weibo&display=popup&client_id=3443163201&response_type=code HTTP/1.1",
			"Host"=>"api.weibo.com",
			"Referer"=>"http://www.ipin.com/index.do?_",
			"Accept"=>"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
			"Accept-Language"=>"zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3",
			"Accept-Encoding"=>"gzip, deflate",
			"Connection"=>"keep-alive",
			"User-Agent"=>"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36"
		}
		puts sss=a.get("https://api.weibo.com/oauth2/authorize?scope=all&redirect_uri=http%3A//www.ipin.com/account/callback/weibo&display=popup&client_id=3443163201&response_type=code",{},heads1)
		puts url2 = sss.header["Location"].join

		heads = {
			"Host"=>"www.ipin.com",
			"Referer"=>"http://www.ipin.com/index.do?_",
			"User-Agent"=>"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36"
		}
		puts a.get(url2,{},heads).body
		return a
		# url1="http://www.ipin.com/index.do"
		# puts a.get(url1).body

	end
	
	def  self.save_text_file(name,file_content)
	   File.open("#{name}", "a+") do |file|
			file.puts file_content
		end
	end
	def  self.get_school_data(a,line)
		
		#sleep(rand(20)+2)
		line = line.gsub("\n","")
		p url = "http://www.ipin.com#{line}"
		#url = "http://www.ipin.com/school/profile/s_52ac2e9b747aec013fcf53a5/"
		school_data = {"url"=>"#{url}"}
		html = a.get(url).body
		doc = Nokogiri::HTML(html)
		page_mod = doc.search("div.page-mod")
		s_score = page_mod.search("div.score")[0]
		sc_page = {}
		page_mod.search("div.f4 li").each do |x|
			p_text = x.inner_text
			p_text = p_text.to_s.split(":")[0]
			bar_dj = x.search("div.bar").inner_text
			bar_width = x.search("div.bar")[0]["style"]
			bar_icon = x.search("i.icon")[0]["v"]
			p sc_page["#{p_text}"] = {"bar_dj"=>bar_dj,"bar_width"=>bar_width,"bar_icon"=>bar_icon}
		end
		school_data.merge!({"sc_page"=>sc_page})
		#p school_data
		p script_s = doc.search("script").to_s
		ppt = script_s.match(/school\/major\/stats",(.+?)\);/)[1]
		ppt_json = JSON.parse(ppt)
		school_data.merge!({"ppt_json"=>ppt_json})
		majors = {}
		doc.search("div.category").each do |x|
			major_category = x.search("div.name")[0].inner_text #大分类
			h = {} 
			x.search("div a").each do |y|
				h["#{y.inner_text}"] = y["href"]
			end
			majors["#{major_category}"] = h
		end
		school_data.merge!({"majors"=>majors})
		save_text_file("ipin_school_page_mod_#{Time.now.year}_#{Time.now.month}_#{Time.now.day}.txt",school_data.to_json)
		
		#p school_data
	end

end

#$redis = Redis.new(:host => '121.40.209.156', :port => '6379', :db => 3)
$redis = Redis.new(:host => '123.196.117.42', :port => '6379', :db => 3)
loop do 
  p ipin_user_count = $redis.scard("ipin_spider_users")
  p url_count = $redis.scard("ipin_spider_scurls")
  p "sdfsdfsdf*****999123123"
  ipin_user = $redis.spop("ipin_spider_users")
  if 0 == ipin_user_count
    break
  end
  if 0 == url_count
    break
  end
  if 2 == ipin_user.split("|").size
  	name = ipin_user.split("|")[0]
    pass = ipin_user.split("|")[1]
    a = nil
    begin 
      a  = Ipin.get_connection("#{name}","#{pass}")
    rescue Exception => e
     p 	ipin_user
     break
    end
    url_strs = $redis.spop("ipin_spider_scurls")
    if  40 >= url_strs.split(",").size
     url_strs.split(",").each do |line|
       begin	
     	Ipin.get_school_data(a,"#{line}")
       rescue Exception => e
		Ipin.save_text_file("ipin_exception_error_#{Time.now.year}_#{Time.now.month}_#{Time.now.day}.txt","#{line}")
		next
	   end
     end
    else
      p "url多余40个" 
      break
    end
  else
    p "账号格式不对"
    break
  end
end
  
  #添加url
  # File.new("./ipin_daxue_url.txt").each_slice(40) do |line|
  #     url_strs = line.collect{|x| x.gsub("\n","")}.join(",")
  #     $redis.sadd("ipin_spider_scurls","#{url_strs}")
  #   end
  #添加账号
  #$redis.sadd("ipin_spider_users","wangsong1949@sina.com|abcd1234")
  #$redis.sadd("ipin_spider_users","13911330243|4604747")
  #----
#   2.1.1 :026 >  File.new("/Users/risyran/shenyong/iease/test_ask_pro/ipin_weibo_user.txt").each do |line|
# 2.1.1 :027 >     line = line.gsub("\t\n","")
# 2.1.1 :028?>   $redis.sadd("ipin_spider_users","#{line}")
# 2.1.1 :029?>   end










